#encoding: utf-8

from urllib import request
from urllib import parse
from http.cookiejar import CookieJar

import pymysql
import json

from lxml import etree
import xlwt
import xlrd
from xlutils.copy import copy
from openpyxl import workbook
import sys
import os


def get_header(referer):
    headers = {
        'Referer': referer,
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
    }
    return headers


def get_opener():
    cookiejar = CookieJar()
    handler = request.HTTPCookieProcessor(cookiejar)
    opener = request.build_opener(handler)
    return opener


def pre_visit(opener, url, headers):
    #url = 'https://www.lagou.com/jobs/list_Java?px=default&city=%E9%87%8D%E5%BA%86'
    req = request.Request(url, headers=headers)
    resp = opener.open(req)
    # print(resp.read().decode('utf-8'))
    with open('./latou.html', 'w', encoding='utf-8') as fp:
        fp.write(resp.read().decode('utf-8'))


def get_data(opener, url, headers, pn, kd):

    #url = 'https://www.lagou.com/jobs/positionAjax.json?px=default&city=%E9%87%8D%E5%BA%86&needAddtionalResult=false'
    form_data = {
        'first': 'true',
        'pn': pn,
        'kd': kd
    }
    req = request.Request(url, data=parse.urlencode(
        form_data).encode('utf-8'), headers=headers)
    resp = opener.open(req)
    res_data = resp.read().decode('utf-8')
    # print(res_data)
    with open('./jobs.txt', 'w', encoding='utf-8') as fp:
        fp.write(res_data)

    json_data = json.loads(res_data)
    totalCount = json_data['content']['positionResult']['totalCount']
    pageSize = json_data['content']['pageSize']
    resultSize = json_data['content']['positionResult']['resultSize']
    print('总条数: ', totalCount, '每页条数：', pageSize, '当前页条数：', resultSize)
    if resultSize > 0:
        print("----------------------")
        #totalCount = totalCount / 15
        # print(json_data['content']['positionResult']['result'])
        results = json_data['content']['positionResult']['result']
        # print(len(results))
        #save_excel(results, (pn - 1) * 15)
        append_excel('./java_lagou_cq.xlsx', results)
        pn += 1
        get_data(opener, url, headers, pn, kd)


def start(pn):
    kd = 'Java'
    ref_url = 'https://www.lagou.com/jobs/list_'
    params = {
        'px': 'default',
        'city': '重庆'
    }
    qs = parse.urlencode(params)
    ref_url = ref_url + kd + "?" + qs

    headers = get_header(ref_url)
    opener = get_opener()
    pre_visit(opener, ref_url, headers)

    data_url = 'https://www.lagou.com/jobs/positionAjax.json?'
    params['needAddtionalResult'] = 'false'
    data_qs = parse.urlencode(params)
    data_url = data_url + data_qs
    # print(data_qs)
    get_data(opener, data_url, headers, pn, kd)


def append_excel(path, contents):
    create_excel(path, False, [])
    workbook = xlrd.open_workbook(path)
    sheets = workbook.sheet_names()
    worksheet = workbook.sheet_by_name(sheets[0])
    rows_old = worksheet.nrows
    new_workbook = copy(workbook)
    new_worksheet = new_workbook.get_sheet(0)
    try:
        index = rows_old + 1
        for obj in contents:
            # print(obj)
            # print('\n')
            cols = [obj['companyId'], obj['companyFullName'], obj['companySize'], obj['financeStage'], str(
                obj['companyLabelList']), obj['industryField'], obj['salary'], obj['district'], obj['positionName'], obj['positionAdvantage'], obj['firstType'], obj['secondType'], obj['thirdType'], obj['subwayline'], obj['linestaion']]
            write_worksheet(new_worksheet, index, cols)
            index += 1
    except:
        print(sys.exec_info()[0])
        pass
    new_workbook.save(path)


def save_excel(contents, index):
    workbook = xlwt.Workbook()
    worksheet = workbook.add_sheet('123')
    try:
        for obj in contents:
            # print(obj)
            # print('\n')
            cols = [obj['companyId'], obj['companyFullName'], obj['companySize'], obj['financeStage'], str(
                obj['companyLabelList']), obj['industryField'], obj['salary'], obj['district'], obj['positionName'], obj['positionAdvantage'], obj['firstType'], obj['secondType'], obj['thirdType'], obj['subwayline'], obj['linestaion']]
            write_worksheet(worksheet, index, cols)
            index += 1
    except:
        print(sys.exec_info()[0])
        pass
    workbook.save('job_datas.xls')


def write_worksheet(worksheet, index, cols):
    colIndex = 0
    for col in cols:
        worksheet.write(index, colIndex, col)
        colIndex += 1


def save_db():
    db_config = {
        'host': '192.168.184.2',
        'port': 3306,
        'user': 'root',
        'passwd': 'rootroot',
        'db': 'boot_plus',
        'charset': 'utf8'
    }

    conn = pymysql.connect(**db_config)
    cursor = conn.cursor()

    query_sql = 'select * from logou_job'
    cursor.execute(query_sql)
    results = cursor.fetchall()
    print(results)


def sheet_method(work_book, add_sheet=[]):
    wk = work_book
    # rename default sheet
    ss_sheet = wk.get_sheet_by_name('Sheet')
    ss_sheet.title = add_sheet[0]

    for i in range(1, len(add_sheet)):
        # add new sheet
        wk.create_sheet(add_sheet[i])

    # switch to active sheet
    sheet_num = wk.get_sheet_names()
    last_sheet = len(sheet_num) - 1
    sheet_index = sheet_num.index(sheet_num[last_sheet])
    wk.active = sheet_index


def create_excel(excel_path="./java_lagou_cq.xlsx", recreate=False, add_sheet=[]):
    """
    判断文件是否存在, 不存在则创建
    """
    # .xls后缀结尾的只能用xlwt模块写，只能写2的16次方即65,535条数据
    # path = os.path.join(os.getcwd()+r'\test.xls')
    # .xlsx的只能用openpyxl模块写，这模块能写2的32次方即4,294,967,296的数据
    #path = os.path.join(os.getcwd() + "\\" + file_name + ".xlsx")
    exist = os.path.exists(excel_path)
    if exist:
        if recreate:
            os.remove(excel_path)
            wk = workbook.Workbook()
            # wheet_method(wk, add_sheet)
            wk.save(excel_path)
        else:
            pass
    else:
        # filename = xlwt.Workbook()
        wk = workbook.Workbook()
        # sheet_method(wk, add_sheet)
        # save workbook
        wk.save(excel_path)
        # print("you had create " + filename)


if __name__ == '__main__':
    print('__main__')
    args = sys.argv
    print(args)
    #start(args[1])
    start(1)
